/*
 LT4EL - Language Technology for eLearning
 
 This library is free software; you can redistribute it and/or modify it under
 the terms of the GNU Lesser General Public License as published by the Free
 Software Foundation; either version 2.1 of the License, or (at your option)
 any later version.
 This library is distributed in the hope that it will be useful, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 details.
 You should have received a copy of the GNU Lesser General Public License along
 with this library; if not, write to the Free Software Foundation, Inc.,
 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */

package org.bultreebank.index.utils;


import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.Vector;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.bultreebank.index.exceptions.FileNotCreatedException;
import org.bultreebank.index.exceptions.IndexSchemaDTDNotFound;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import clark.dom.impl.DOMFactory;
import clark.dtd.DTD;
import clark.dtd.DTDFactory;
import clark.loader.DOMPrinter;
import clark.loader.DefaultFileFilter;
import clark.loader.XMLParser;
import clark.xpath.MainXPath;

/**
 * Basic utility class
 * @author Aneliya Ticheva
 *
 */
public class Utils implements Constants{

	public static String getStringOfStringArray(String[] array) 
	{
		StringBuffer buf = new StringBuffer();
		for (int i = 0; i < array.length; i++)
			buf.append(array[i] + WHITESPACE);
		return buf.toString();
	}
	public static boolean deleteFileOrDirectory(String filePath) 
	{
		File f = new File(filePath);
		if (f.exists())
		{
			 if (f.isDirectory())
			 {
				 File[] files = f.listFiles();
				 for (File file : files)
					 deleteFileOrDirectory(file.getAbsolutePath());
			 }
			 return f.delete();
		}
		return true;
	}
	public static void setAutogeneratedIDs(Document sourceDoc, 
			String absoluteXPATHSelection, String attrName,
			String sourceValue)
			throws Exception
	{
		Element docElement = sourceDoc.getDocumentElement();
		List resNodeList = MainXPath.processXPath(absoluteXPATHSelection,
      		   docElement);
		int i = 1;
		for (Object elem:resNodeList)
		{
			String generatedID = sourceValue + i; 
			((Element) elem).setAttribute(attrName, generatedID);
			i++;
		}
		 
	}
	 /**
     * For a given list of concepts, normalization means a string, 
     * that contains all the 
     * concepts from the list.
     * @param data
     * @return
     */
    public static String getTokenStringFromStringsList(List<String> data){
        StringBuffer buf = null;
        if (data != null){
            buf = new StringBuffer();
            for (Iterator j = data.iterator(); j.hasNext();){
                String temp = (String)j.next();
                buf.append(temp + WHITESPACE);
            }
        }
        return buf.toString();
    }
    public static void loadTableFile(Properties systemIDs_realIDs, 
    		Properties realIDs_systemIDs, String path)
	throws FileNotCreatedException,IOException
	{
		
		File f = Utils.createNewFile(path);
		if (f == null)
			throw new FileNotCreatedException("The file: " + path +
					" is not created!");
		BufferedReader reader = new BufferedReader(new FileReader(
				f));
		String line = reader.readLine();
		while (line != null)
		{
			int i = line.indexOf(WHITESPACE);
			String internalIndexID = line.substring(0, i);
			String realIndexID = line.substring(i + 1);
			systemIDs_realIDs.put(internalIndexID, realIndexID);
			realIDs_systemIDs.put(realIndexID, internalIndexID);
			line = reader.readLine();
		}
		reader.close();
	}
    public static void saveTableFile(String path, Properties systemID_realID)
    throws Exception
	{
		Utils.createNewFile(path);
		PrintWriter out = new PrintWriter(new BufferedWriter(new 
				FileWriter(path)));
		Enumeration indexIDs = systemID_realID.keys();
		while(indexIDs.hasMoreElements())
		{
			String indexID = indexIDs.nextElement().toString();
			if (indexID != null)
			{
				String realID = systemID_realID.getProperty(indexID);
				String line = indexID + WHITESPACE + realID + NEW_LINE;
				out.print(line);
				out.flush();
			}
		}
		out.close();
	}
    public static Document removeNodesFromDoc(Document doc,
    		List<String> needed)
    throws Exception
    {
    	Element root = doc.getDocumentElement();
    	NodeList list = root.getChildNodes();
    	for (int i = 0; i < list.getLength(); i++)
    	{
    		Element document = (Element)list.item(i);
    		String docID = document.getAttribute(ID_ATTRIBUTE);
    		if (!needed.contains(docID))
    		{
    			root.removeChild(document);
    		}
    	}
    	return doc;
    }
    
	/**
	 * Creates a new XML document and returns it.
	 * @return a new XML document without any data
	 * @exception ParserConfigurationException if an error occurred creating 
	 * the new XML document
	 */
	public static Document newDocument() throws ParserConfigurationException
	{
	     DocumentBuilderFactory factory;
	     DocumentBuilder builder;
	     factory = DocumentBuilderFactory.newInstance();
	     factory.setNamespaceAware(false);
	     factory.setValidating(false);
	     builder = factory.newDocumentBuilder();
	     return builder.newDocument();

	}
	
	/**
	 * Checks if in the nodes list there is an attribute <attributeName> with 
	 * value <attributeValue>
	 * @param attributeValue - the value of the attribute to be searched for
	 * @param nodes - a List of Nodes with attribute <attributeName>
	 * @param attributeName - the name of the attribute
	 * @return true if some node from  the nodes list has attribute <attributeName>
	 * with value <attributeValue>
	 * @throws Exception in case of problems
	 */
	public static boolean checkIfInNodeListAsAttributeValue(
			String attributeValue,
			NodeList nodes,
			String attributeName)
	throws Exception
	{
		for (int i = 0; i < nodes.getLength(); i++)
		{
			String id = ((Element)nodes.item(i)).getAttribute(attributeName);
			if (id.equals(attributeValue))
				return true;
		}
		return false;
	}
	

	public static String getMaxDayForManth(String month, String year)
	throws Exception
	{
		if (month.charAt(0) == '0')
		{
			char c =  month.charAt(1);
			Integer i = Integer.valueOf(String.valueOf(c));
			int monthInt = i.intValue();
			int yearInt = Integer.valueOf(year).intValue();
			boolean isHighStem = false;
			if ((yearInt % 4 == 0 )|| (yearInt % 100 == 0 && yearInt % 400 == 0))
				isHighStem = true;
			
			switch (monthInt)
			{
				case 1: case 3: case 5: case 7: case 8: return String.valueOf(31);
				case 2:
				{
					if (isHighStem) 
						return  String.valueOf(29);
					return  String.valueOf(28);
				}
				case 4: case 6:  case 9: return String.valueOf(30);  
				default: throw new Exception("Invalid date!");
			}
		}
		else if (month.charAt(0) == '1')
		{
			if (month.charAt(1) == '1' )
				return  String.valueOf(30);
			else if (month.charAt(1) == '2' || month.charAt(1) == '0')
				return  String.valueOf(31);
			else
				throw new Exception("Invalid date!");
		}
		throw new Exception("Invalid date!");
	}
	
	public static String normalizeFromDate(String date) throws Exception
	{
		StringTokenizer st = new StringTokenizer(date, "/,\\,-,:");
		int datePartsNum = st.countTokens();
		if (datePartsNum < 0 || datePartsNum > 3)
			throw new Exception("The date is invalid!");
		switch (datePartsNum)
		{
			
			case 1:
			{
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				return year + "/01/01";
			}
			case 2:
			{
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				String month = st.nextToken();
				if (month.length() != 2)
					throw new Exception("The date is invalid!");
				return year + "/" + month + "/" + "01";
			}
				
			case 3:	
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				String month = st.nextToken();
				if (month.length() != 2)
					throw new Exception("The date is invalid!");
				String day = st.nextToken();
				if (day.length() != 2)
					throw new Exception("The date is invalid!");
				
				return year + "/" + month + "/" + day; 
		}
		
		return "";
	}
	
	public static String normalizeToDate(String date) throws Exception
	{
		StringTokenizer st = new StringTokenizer(date, "/,\\,-,:");
		int datePartsNum = st.countTokens();
		if (datePartsNum < 0 || datePartsNum > 3)
			throw new Exception("The date is invalid!");
		switch (datePartsNum)
		{
			
			case 1:
			{
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				return year + "/12/31";
			}
			case 2:
			{
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				String month = st.nextToken();
				if (month.length() != 2)
					throw new Exception("The date is invalid!");
				return year + "/" + month + "/" + getMaxDayForManth(month, year);
			}
				
			case 3:	
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				String month = st.nextToken();
				if (month.length() != 2)
					throw new Exception("The date is invalid!");
				String day = st.nextToken();
				if (day.length() != 2)
					throw new Exception("The date is invalid!");
				
				return year + "/" + month + "/" + day; 
		}
		
		return "";
	}

	public static String normalizeMiddleDate(String date) throws Exception
	{
		StringTokenizer st = new StringTokenizer(date, "/,\\,-,:");
		int datePartsNum = st.countTokens();
		if (datePartsNum < 0 || datePartsNum > 3)
			throw new Exception("The date is invalid!");
		switch (datePartsNum)
		{
			
			case 1:
			{
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				return year + "/06/15";
			}
			case 2:
			{
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				String month = st.nextToken();
				if (month.length() != 2)
					throw new Exception("The date is invalid!");
				return year + "/" + month + "/" + "15";
			}
				
			case 3:	
				String year = st.nextToken();
				if (year.length() != 4)
					throw new Exception("The date is invalid!");
				String month = st.nextToken();
				if (month.length() != 2)
					throw new Exception("The date is invalid!");
				String day = st.nextToken();
				if (day.length() != 2)
					throw new Exception("The date is invalid!");
				
				return year + "/" + month + "/" + day; 
		}
		
		return "";
	}

	public static Element getNodeFromListByAttributes(
			Map<String, String> attr,
			NodeList nodes
			)
	throws Exception
	{
		for (int i = 0; i < nodes.getLength(); i++)
		{
			Set<String> keys = attr.keySet();
			boolean fix = false;
			for (Iterator iter = keys.iterator(); iter.hasNext(); )
			{
				String attr_name = (String)iter.next();
				String attr_value = attr.get(attr_name);
				String id = ((Element)nodes.item(i)).getAttribute(attr_name);
				if (!id.equals(attr_value))
				{
					fix = false;
					break;
				}
				else
					fix = true;
			}
			if (!fix)
				continue;
			else
				return (Element)nodes.item(i);
			
		}
		return null;
	}
	/**
	 * Returns a List of strings, created by the separate tokens in the
	 * <tokenString>
	 * @param tokenString - string of tokens separated with whitespace
	 * @return
	 * @throws Exception
	 */
	public static List<String> getListFromTokenizedString(
			String tokenString)
	throws Exception
	{
		List<String> fields = new ArrayList<String>();
		StringTokenizer st = new StringTokenizer(tokenString);
		while (st.hasMoreTokens())
		{
			String field = st.nextToken();
			fields.add(field);
		}
		return fields;
	}
	
	/**
	 * Returns the first node from the nodeList that has attribute <attributeName>
	 * with value <attributeValue>
	 * @param attributeValue - the value of the attribute to be searched for
	 * @param nodes - a List of Nodes with attribute <attributeName>
	 * @param attributeName - the name of the attribute
	 * @return
	 * @throws Exception
	 */
	public static Node getNodeFromNodeList(String attributeValue,
			NodeList nodes, String attributeName)
	throws Exception
	{
		for (int i = 0; i < nodes.getLength(); i++)
		{
			String id = ((Element)nodes.item(i)).getAttribute(attributeName);
			if (id.equals(attributeValue))
				return nodes.item(i);
		}
		return null;
	}
	public static Element getNodeFromNodeVector(String attributeValue,
			Vector<Element> nodes, String attributeName)
	throws Exception
	{
		for (int i = 0; i < nodes.size(); i++)
		{
			String id = (nodes.get(i)).getAttribute(attributeName);
			if (id.equals(attributeValue))
				return nodes.get(i);
		}
		return null;
	}
	public static List<Node> getAllNodesFromNodeList(String attributeValue,
			NodeList nodes, String attributeName)
	throws Exception
	{
		List<Node> res = new ArrayList<Node>();
		for (int i = 0; i < nodes.getLength(); i++)
		{
			String id = ((Element)nodes.item(i)).getAttribute(attributeName);
			if (id.equals(attributeValue))
				 res.add(nodes.item(i));
		}
		return res;
	}
	public static Vector<Element> getAllNodesWithAttribute(
			String attributeName,
			NodeList nodes)
	throws Exception
	{
		Vector<Element> res = new Vector<Element>();
		for (int i = 0; i < nodes.getLength(); i++)
		{
			String id = ((Element)nodes.item(i)).getAttribute(attributeName);
			if (id != null && !id.equals(EMPTY_STR))
				 res.add((Element)nodes.item(i));
		}
		return res;
	}
	public static String getStringFromNodeList(boolean searchable,
			List<Node> nodes)
	{
		StringBuffer buf = new StringBuffer();
		for (Node node:nodes)
		{
			 if (searchable)
            	 buf.append(
        			 Utils.normalizeTerm(node.getNodeValue()).
            			 toLowerCase()
            			 +
	            		 WHITESPACE);
             else
            	 buf.append(node.getNodeValue() + WHITESPACE);
		}
		return buf.toString();
	}
	/**
	 * Returns a string representation of the XML document
	 * @param doc - a DOM representation of the XML document
	 * @return
	 * @throws Exception
	 */
	public static String getStringRepresentationOfDocument(Document doc)
    throws Exception
    {
	 	
		ByteArrayOutputStream out = new ByteArrayOutputStream();
		PrintStream pr = new PrintStream(out);
		if (doc.getDocumentElement() == null)
			return EMPTY_STR;
		DOMPrinter.printDOM(doc.getDocumentElement(), pr,null, null,
				DefaultFileFilter.convertEncoding(CHAR_SET_UTF8));
		
    	String result = out.toString(CHAR_SET_UTF8);
    	
    	pr.close();
    	out.close();
    	
    	return result;
    }
	
	public static List<String> unionSets(List<String> l1, List<String> l2)
	{
		int n = l1.size(), m = l2.size();
		boolean ifSecondLonger = false;
		if (n < m)
		{
			n = m;
			ifSecondLonger = true;
		}
		if (ifSecondLonger)
		{
			for (Iterator i = l2.iterator(); i.hasNext(); )
			{
				String elem = (String)i.next();
				if (!l1.contains(elem))
					l1.add(elem);
			}
			return l1;
		}
		else
		{
			for (Iterator j = l1.iterator(); j.hasNext(); )
			{
				String elem = (String)j.next();
				if (!l2.contains(elem))
					l2.add(elem);
			}
			return l2;

		}
	}
	
	public static List<String> intersectionSets(List<String> l1, List<String> l2)
	{
		for (Iterator<String> iter = l1.iterator(); iter.hasNext(); )
		{

			if (!l2.contains(iter.next()))
				iter.remove();
		}
		return l1;
	}

	public static List<String> substractionSets(List<String> l1, List<String> l2)
	{
		for (Iterator<String> iter = l1.iterator(); iter.hasNext(); )
		{

			if (l2.contains(iter.next()))
				iter.remove();
		}
		return l1;
	}

	public static void saveDocument(String path,String dtdContent, Document content)
	 throws Exception
	 {
		 File f = Utils.createNewFile(path);
		 if (f == null)
			throw new Exception ("The file: " + path + " is not created!");
		 FileOutputStream out = new FileOutputStream(f);
		 PrintStream pr = new PrintStream(out);
		 DOMPrinter.printDOM(content.getDocumentElement(),
				 pr,dtdContent, DefaultFileFilter.UTF_8);
	}

	public static void saveDocument(String path,Document content, String encoding)
	 throws Exception
	 {
		 File f = Utils.createNewFile(path);
		 if (f == null)
			throw new Exception ("The file: " + path + " is not created!");
		 FileOutputStream out = new FileOutputStream(f);
		 String docStr = Utils.getStringRepresentationOfDocument(content);
		 OutputStreamWriter writer = new OutputStreamWriter(out, encoding);
		 writer.write(docStr);
		 writer.flush();
		 writer.close();
	}
	/**
	 * Saves the file content to the file reachable by the speciifed path param
	 * @param path - the path to the file
	 * @param content - the file content
	 * @throws Exception
	 */
	public static void saveFile(String path, String content)
	 throws Exception
	 {
		 File f = Utils.createNewFile(path);
		 if (f == null)
			throw new Exception ("The file: " + path + " is not created!");
		 FileOutputStream out = new FileOutputStream(path);  
		 OutputStreamWriter writerOutput = new OutputStreamWriter(out,
				 CHAR_SET_UTF8);
		 writerOutput.write(content);
		 writerOutput.flush();
		 out.close();
		 writerOutput.close();
		
	}
	
	
	
	public static DTD loadDTD(String path) throws Exception
	{
		File f = new File(path);
		if (!f.exists())
			throw new IndexSchemaDTDNotFound("The DTD for validation: " 
					+ path + " doesn't exist! ");
		return DTDFactory.createDTDFromFile(path,
				DefaultFileFilter.UTF_8);
	}
	/**
	 * Deletes a file reachable by the path parameter 
	 * @param path - the path to the file to be deleted
	 * @return true if the delete operation is successful and false otherwise
	 * @throws Exception
	 */
	public static boolean deleteFile(String path)
	throws Exception
	{
		 File f = new File(path);
		 if (f.exists())
			 return f.delete();
		 return true;
	    
	}
	
	
	/**
     * For a given list of concepts, normalization means a string, that contains all the 
     * concepts from the list, but changed to lowercase, with removed ":" characters and 
     * separated with spaces (This is for preparing an appropriate input for the Lucene 
     * search engine)
     * @param data
     * @return
     */
    public static String normalizeList(List<String> data){
        StringBuffer buf = null;
        if (data != null){
            buf = new StringBuffer();
            for (int j = 0; j < data.size(); j++){
                String temp = data.get(j);
                temp = Utils.normalizeTerm(temp).toLowerCase();
                buf.append(temp + WHITESPACE);
            }
        }
        return buf.toString();
    }
    
	public static String normalizeTerm(String term)
	{ /*
		if (term.startsWith(EXCLUDED))
		{
			term = EXCLUDED + term.substring(1).replace(EXCLUDED, EMPTY_STR).
			replace(COLUMN, EMPTY_STR).replace(LEFT_SLASH, EMPTY_STR).
			replace(FULL_STOP, EMPTY_STR).replace(INCLUDED, EMPTY_STR);
		}
		else if (term.startsWith(INCLUDED))
		{
			term = INCLUDED + term.substring(1).replace(INCLUDED, EMPTY_STR).
			replace(EXCLUDED, EMPTY_STR).replace(COLUMN, EMPTY_STR).
			replace(LEFT_SLASH, EMPTY_STR).replace(FULL_STOP, EMPTY_STR);;
		}
		else
		{
			term = term.replace(EXCLUDED, EMPTY_STR).replace(INCLUDED, EMPTY_STR).
			replace(COLUMN, EMPTY_STR).replace(LEFT_SLASH, EMPTY_STR).
			replace(FULL_STOP, EMPTY_STR);
		}
		if (term.endsWith(SIMILAR_SPELLING_SIGN))
		{
			term = term.substring(0, term.lastIndexOf(SIMILAR_SPELLING_SIGN)).
			replace(SIMILAR_SPELLING_SIGN, EMPTY_STR).replace(DISTANCE_SIGN,
					EMPTY_STR)+ SIMILAR_SPELLING_SIGN;
		}
		else if (term.startsWith(DISTANCE_SIGN))
		{
			term = term.substring(0, term.lastIndexOf(DISTANCE_SIGN)).
			replace(DISTANCE_SIGN, EMPTY_STR).
			replace(SIMILAR_SPELLING_SIGN, EMPTY_STR) + DISTANCE_SIGN;
		}
		else
		{*/
			term = term.replace(DISTANCE_SIGN, EMPTY_STR).replace(
					SIMILAR_SPELLING_SIGN, 
					EMPTY_STR).replace(EXCLAMATION_MARK, EMPTY_STR).replace(LEFT_SLASH, EMPTY_STR).
					replace(LEFT_BRACKET, EMPTY_STR).replace(RIGHT_BRACKET, EMPTY_STR).
					replace(CURLY_LEFT_BRACKET, EMPTY_STR).replace(CURLY_RIGHT_BRACKET, EMPTY_STR).
					replace(INCLUDED, EMPTY_STR).replace(EXCLUDED, EMPTY_STR).replace(COLUMN, EMPTY_STR);
		//}
			
		return term;//.toLowerCase();
	}

	/**
	 * Gets the document ID for the file
	 * @param metaDataXML - the metadata file for a document to be indexed
	 * @return
	 * @throws Exception
	 */
	public static String getDocIDFromMetadataFile(String metaDataXML)
	throws Exception
	{
		Document metaDOC = Utils.getDocumentFromString(metaDataXML);
		NamedNodeMap attrs = metaDOC.getAttributes();
		Node ID = attrs.getNamedItem(DOC_ID);
		if (ID != null)
		{
			return ID.getNodeValue();
		}
		return null;
	}
	
	/**
	 * Returns a string representation of the file content of the file reachable
	 * by the specified path parameter
	 * @param path - the path to the file
	 * @return
	 * @throws FileNotFoundException
	 * @throws UnsupportedEncodingException
	 * @throws IOException
	 */
	public static String getStringRepresentationOfFileContent(String path)
	throws FileNotFoundException, UnsupportedEncodingException, IOException,
	Exception
	{
		return getStringRepresentationOfFileContent(path, CHAR_SET_UTF8);
	}
	public static String getStringRepresentationOfFileContent(String path, String encoding)
	throws FileNotFoundException, UnsupportedEncodingException, IOException,
	Exception
	{
		FileInputStream fis = null;
		try
		{
			fis = new FileInputStream(path);
		}
		catch (FileNotFoundException e)
		{
			throw new Exception ("The file " + path + " doesn't exist!");
		}
		BufferedReader reader = new BufferedReader(new InputStreamReader(
					fis, encoding));
		
		char[] buffer = new char[BUFF_SIZE];
		StringBuffer fileString = new StringBuffer();
		int readChars = BUFF_SIZE;
		while (readChars == BUFF_SIZE && readChars != END_OF_FILE)
		{
			try{
			    readChars = reader.read(buffer);
			}
			catch(IOException e)
			{
				e.printStackTrace(System.err);
			}
            if (readChars != END_OF_FILE)
                fileString.append(buffer,0,readChars);
		}
		reader.close();
		return  fileString.toString();
	}
	public static Set<String> getSetFromString(String input)
	{
		if (input != null)
		{
			Set<String> result = new HashSet<String>();
			StringTokenizer st = new StringTokenizer(input);
			while (st.hasMoreTokens())
				result.add(st.nextToken());
		return result;
	
		}
		return new HashSet<String>();
	}
	public static File createNewFolder(String path)
	throws FileNotCreatedException
	{
		File f = new File(path);
		if (!f.exists() || !f.isDirectory())
			if (f.mkdir())
				return f;
			else throw new FileNotCreatedException("The directory: " + path + " is not created!");
		else 
			return f;
				//throw new Exception("The directory: " + path + " is not created!");
	}
	public static File createNewFile(String path)
	throws FileNotCreatedException, IOException
	{
		File f = new File(path);
		if (!f.exists())
			if (f.createNewFile())
				return f;
			else
				return null;
		else 
			return f;
	}
	public static Document getDocumentFromString(String XMLDoc) throws Exception
	{
		 XMLParser parser = new XMLParser();
	     Hashtable buf = new Hashtable();
	     Document document = DOMFactory.getEmptyDocument();
	     int pos = parser.parseHeader(XMLDoc, buf);
	     parser.parse(document, XMLDoc, null, pos);
	     return document;
	     
	}
	
	public static int docLength(Document doc)
	{
		String xpath = "//text()";
		List<Node> textNodes = MainXPath.processXPath(xpath, doc.getDocumentElement());
		int numTokens = 0;
		for (Node n:textNodes)
		{
			String value  = n.getNodeValue();
			StringTokenizer st = new StringTokenizer(value);
			numTokens += st.countTokens();
		}
		return numTokens;
	}
	/**
	 * Gets a valid internal docID for the real(explicit) docID
	 * @param docID
	 * @return the internal docID
	 * @throws Exception
	 */
	public static String getAvailableInternalID(String prefix, String docID,
			Properties table)
	throws Exception
	{
		String internalName = null;
		Collection values = table.values();
		if (values.size() == 0)
		{
			internalName = prefix + 1;
			return internalName;
		}
		if (docID != null && table.containsKey(docID))
			return table.getProperty(docID);
		int[] numValies = new int[values.size()];
		int i = 0;
		int prefixLenght = prefix.length();
		for (Iterator<String> iter = values.iterator(); iter.hasNext(); )
		{
			numValies[i] = Integer.parseInt(iter.next().substring(
					prefixLenght));
			i++;
		}
		Arrays.sort(numValies); 
		for (int j = 0; j < numValies.length; j++)
		{
			if (j + 1 != numValies[j])
			{
				internalName = prefix + (j + 1);
				break;
			}
			else if (j == numValies.length - 1)
			{
				internalName = prefix + (j + 2);
				break;
			}
		}
		return internalName;
	
	}
	
	
}
